import scrapy


class BiqugeFllowSpider(scrapy.Spider):
    name = "biquge_fllow"

    def start_requests(self):
        urls = 'www.biquge5200.com/book/'
        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse,)

    def parse(self, response):
        for href in response.css('div.booklist > ul > li > a::attr(href)'):
            url = response.urljoin(href.extract())
            yield scrapy.Request(url, callback=self.parse_dir_contents)


    def parse_dir_contents(self, response):
        for sel in response.css('div.contentbox'):
            item = {}
            item['title'] = sel.css('div.bookname > h1::text').extract()
            item['content'] = sel.css('div.content > div.content-text::text').extract()
            yield item
            next_page = response.css('div.bottem1 > a:nth-child(4)::attr(href)').extract_first()


